All data used by this script was gathered using the Harmony Rosetta-CLI fork using the parse:supply command.
The script was written quickly and is NOT optimal for parsing the data, nor is it memory friendly. If you wish to run this script yourself, you have been warned.
import numpy as np
import json
import os
import plotly.graph_objects as go
from collections import defaultdict
import warnings; warnings.simplefilter('ignore')
from IPython.display import clear_output
SHARD = 0
STAKING_EPOCH = 186
GRAPH_DIR = "./graphs"
if not os.path.exists(GRAPH_DIR):
os.makedirs(GRAPH_DIR)
DATA_DIR = "/Users/danielvdm/go/src/github.com/Daniel-VDM/jupyter/hmy_chain_parse"
DATA_FILE_PREFIX = "parse_output"
ACC_FILE_PREFIX = "parse_last_seen_account"
def load_data(shard=0):
data_dir = f"{DATA_DIR}/s{shard}/parse"
data_files= [f for f in os.listdir(data_dir) if f.startswith(DATA_FILE_PREFIX)]
data_files.sort(key=lambda e: int(e.split(">")[-1]))
acc_files = [f for f in os.listdir(data_dir) if f.startswith(ACC_FILE_PREFIX)]
assert len(acc_files) > 0
blocks, unique_accs = [], set()
for path in data_files:
blk_path = f"{data_dir}/{path}"
with open(blk_path, 'r') as f:
clear_output(wait=True)
print(f"loading block data from: {blk_path}")
for blk in f:
blocks.append(json.loads(blk))
clear_output(wait=True)
print(f"sorting loaded block data...")
blocks.sort(key=lambda e: e["block_identifier"]["index"])
for path in acc_files:
acc_path = f"{data_dir}/{path}"
with open(acc_path, 'r') as f:
clear_output(wait=True)
print(f"loading account data from: {acc_path}")
for acc in f:
if acc.startswith("one1"):
unique_accs.add(acc)
clear_output(wait=True)
print(f"Loaded data for shard {shard} successfully!")
return {
"blocks": np.array(blocks),
"unique_accounts": unique_accs,
}
data = load_data(SHARD)
Loaded data for shard 0 successfully!
data["blocks"] is a list of Block JSON objects (stored as a dict) order by block index.
The Block object looks like the following:
{
"block_identifier": {
"index": 0,
"hash": "0xb4d158b82ac8a653c42b78697ab1cd0c6a0d9a15ab3bc34130f0b719fb174d2a"
},
"block_epoch": 0,
"approx_block_rewards_so_far": 0,
"approx_circulating_supply_so_far": 1.26e+28,
"block_rewards": 0,
"total_amount_deducted_from_accounts": 0,
"total_amount_credited_from_accounts": 1.26e+28,
"number_of_contracts_created": 0,
"number_of_transactions": 1,
"number_of_accounts": 1,
"amount_transferred": 0,
"cross_shard_amount_received": 0,
"cross_shard_amount_sent": 0,
"gas_fees": 0,
"approx_number_of_unique_accounts_so_far": 1
}
def as_one(val):
""" Converts val (in ATTO) to ONE.
"""
return np.multiply(val, 1e-18)
def get_blk_index(b):
"""
Get the block's index
"""
return b["block_identifier"]["index"]
def get_stats_by_epoch(blocks, starting, ending):
""" Calculate stats from the starting epoch (inclusive) to the ending epoch (exclusive)
"""
def include(blk):
return starting <= blk["block_epoch"] < ending
print(f"calculating stats from epoch [{starting}, {ending})")
return get_basic_stats(blocks, include)
def get_stats_by_block(blocks, starting, ending):
""" Calculate stats from the starting block (inclusive) to the ending block (exclusive)
"""
def include(blk):
return starting <= blk["block_identifier"]["index"] < ending
print(f"calculating stats from block [{starting}, {ending})")
return get_basic_stats(blocks, include)
pre_staking_base_block_rewards = 24000000000000000000 # In ATTO
def get_basic_stats(blocks, inclusion_fn=lambda blk: True):
"""
Goes through all of the given blocks and calculates the basic stats for
every block that passes the given inclusion function.
"""
counters = defaultdict(int)
max_epoch, min_epoch, max_block, min_block = -float("inf"), float("inf"), -float("inf"), float("inf")
max_tx_per_block, max_tx_per_block_id = -float("inf"), None
max_accs_per_block, max_accs_per_block_id = -float("inf"), None
min_tx_per_block, min_tx_per_block_id = float("inf"), None
min_accs_per_block, min_accs_per_block_id = float("inf"), None
for blk in blocks:
if not inclusion_fn(blk):
continue
counters["block_rewards"] += blk["block_rewards"]
counters["block_count"] += 1
counters["gas_fees"] += blk["gas_fees"]
tx_count = blk["number_of_transactions"]
if blk["block_epoch"] < STAKING_EPOCH:
counters["coinbase_rewards"] += max(blk["block_rewards"] - pre_staking_base_block_rewards, 0)
tx_count -= 1 # Do not count side effect tx since each block has one.
counters["transaction_count"] += tx_count
counters["cross_shard_incomming_funds"] += blk["cross_shard_amount_received"]
counters["cross_shard_outgoing_funds"] += blk["cross_shard_amount_sent"]
counters["contract_creations"] += blk["number_of_contracts_created"]
if tx_count > max_tx_per_block:
max_tx_per_block = tx_count
max_tx_per_block_id = blk["block_identifier"]
if tx_count < min_tx_per_block:
min_tx_per_block = tx_count
min_tx_per_block_id = blk["block_identifier"]
if blk["number_of_accounts"] > max_accs_per_block:
max_accs_per_block = blk["number_of_accounts"]
max_accs_per_block_id = blk["block_identifier"]
if blk["number_of_accounts"] < min_accs_per_block:
min_accs_per_block = blk["number_of_accounts"]
min_accs_per_block_id = blk["block_identifier"]
counters["total_deducted_amount"] += blk["total_amount_deducted_from_accounts"]
counters["total_credited_amount"] += blk["total_amount_credited_from_accounts"]
counters["total_supply"] += (blk["total_amount_credited_from_accounts"] - blk["total_amount_deducted_from_accounts"])
max_epoch = max(max_epoch, blk["block_epoch"])
max_block = max(max_block, blk["block_identifier"]["index"])
min_epoch = min(min_epoch, blk["block_epoch"])
min_block = min(min_block, blk["block_identifier"]["index"])
return {
"funds": {
"block_rewards": as_one(counters["block_rewards"]),
"gas": as_one(counters["gas_fees"]),
"coinbase_rewards": as_one(counters["coinbase_rewards"]),
"cross_shard_received": as_one(counters["cross_shard_incomming_funds"]),
"cross_shard_sent": as_one(counters["cross_shard_outgoing_funds"]),
"supply": {
"circulating_contribution": as_one(counters["total_supply"]),
"deducted_from_accounts": as_one(counters["total_deducted_amount"]),
"credited_to_accounts": as_one(counters["total_credited_amount"]),
},
},
"counts": {
"blocks": counters["block_count"],
"contract_creations": counters["contract_creations"],
"transaction": counters["transaction_count"],
},
"max": {
"transaction_per_block": {
"count": max_tx_per_block,
"identifier": max_tx_per_block_id,
},
"accounts_per_block": {
"count": max_accs_per_block,
"identifier": max_accs_per_block_id,
},
"seen_epoch": max_epoch,
"seen_block": max_block,
},
"min": {
"transaction_per_block": {
"count": min_tx_per_block,
"identifier": max_tx_per_block_id,
},
"accounts_per_block": {
"count": min_accs_per_block,
"identifier": max_accs_per_block_id,
},
"seen_epoch": min_epoch,
"seen_block": min_block,
}
}
def get_high_level_stats():
pre_staking_stats = get_stats_by_epoch(data["blocks"], 0, STAKING_EPOCH)
overall = get_stats_by_epoch(data["blocks"], 0, data["blocks"][-1]["block_epoch"]+1)
# staking_stats includes pre-steaking epoch (STAKING_EPOCH-1) for clearer staking stats
staking_stats = get_stats_by_epoch(data["blocks"], STAKING_EPOCH-1, data["blocks"][-1]["block_epoch"]+1)
clear_output(wait=True)
print("Pre-staking stats: " + json.dumps(pre_staking_stats, sort_keys=True, indent=4))
print()
print("Staking stats: " + json.dumps(staking_stats, sort_keys=True, indent=4))
print()
print("Overall stats: " + json.dumps(overall, sort_keys=True, indent=4))
get_high_level_stats()
Pre-staking stats: {
"counts": {
"blocks": 3375104,
"contract_creations": 298,
"transaction": 29865
},
"funds": {
"block_rewards": 81002455.45939165,
"coinbase_rewards": 7.4593916560889575,
"cross_shard_received": 44736815.82998507,
"cross_shard_sent": 510842.3964620733,
"gas": 7.508740156039545,
"supply": {
"circulating_contribution": 12290397575.305397,
"credited_to_accounts": 40792334548.28412,
"deducted_from_accounts": 28501936972.978714
}
},
"max": {
"accounts_per_block": {
"count": 250,
"identifier": {
"hash": "0xd8da3c30639d8c40535ed70e766a872d48c68107903ae30411050f84e1829209",
"index": 3375103
}
},
"seen_block": 3375103,
"seen_epoch": 185,
"transaction_per_block": {
"count": 10,
"identifier": {
"hash": "0x4b849d731c892cd66c24b8ba557ba0e1e3ea5c54e88d46f7f4f729f134219a68",
"index": 2171021
}
}
},
"min": {
"accounts_per_block": {
"count": 0,
"identifier": {
"hash": "0xd8da3c30639d8c40535ed70e766a872d48c68107903ae30411050f84e1829209",
"index": 3375103
}
},
"seen_block": 0,
"seen_epoch": 0,
"transaction_per_block": {
"count": 0,
"identifier": {
"hash": "0x4b849d731c892cd66c24b8ba557ba0e1e3ea5c54e88d46f7f4f729f134219a68",
"index": 2171021
}
}
}
}
Staking stats: {
"counts": {
"blocks": 1950486,
"contract_creations": 1018,
"transaction": 286047
},
"funds": {
"block_rewards": 156854167.18313855,
"coinbase_rewards": 1.0781527224483465,
"cross_shard_received": 204996726.78385597,
"cross_shard_sent": 3016271.0499627236,
"gas": 82.52424866423968,
"supply": {
"circulating_contribution": -3685409073.835393,
"credited_to_accounts": 39731682958.947525,
"deducted_from_accounts": 43417092032.78291
}
},
"max": {
"accounts_per_block": {
"count": 1083,
"identifier": {
"hash": "0xef8b9c9a45ef8f2f6f151610de3b766bd07e1c5ccd91e494cde1e8dd37f91778",
"index": 3555327
}
},
"seen_block": 5309205,
"seen_epoch": 304,
"transaction_per_block": {
"count": 14,
"identifier": {
"hash": "0x4480b45205b1d5555d280f33f97bbf37fe5efc1ecac71b364b2212eb8a84d1e9",
"index": 3767045
}
}
},
"min": {
"accounts_per_block": {
"count": 0,
"identifier": {
"hash": "0xef8b9c9a45ef8f2f6f151610de3b766bd07e1c5ccd91e494cde1e8dd37f91778",
"index": 3555327
}
},
"seen_block": 3358720,
"seen_epoch": 185,
"transaction_per_block": {
"count": 0,
"identifier": {
"hash": "0x4480b45205b1d5555d280f33f97bbf37fe5efc1ecac71b364b2212eb8a84d1e9",
"index": 3767045
}
}
}
}
Overall stats: {
"counts": {
"blocks": 5309206,
"contract_creations": 1316,
"transaction": 311311
},
"funds": {
"block_rewards": 237463405.5643775,
"coinbase_rewards": 7.4593916560889575,
"cross_shard_received": 232122851.0097409,
"cross_shard_sent": 3504013.2640097965,
"gas": 88.90548759788028,
"supply": {
"circulating_contribution": 9021838540.176445,
"credited_to_accounts": 80011297259.45395,
"deducted_from_accounts": 70989458719.27751
}
},
"max": {
"accounts_per_block": {
"count": 1083,
"identifier": {
"hash": "0xef8b9c9a45ef8f2f6f151610de3b766bd07e1c5ccd91e494cde1e8dd37f91778",
"index": 3555327
}
},
"seen_block": 5309205,
"seen_epoch": 304,
"transaction_per_block": {
"count": 14,
"identifier": {
"hash": "0x4480b45205b1d5555d280f33f97bbf37fe5efc1ecac71b364b2212eb8a84d1e9",
"index": 3767045
}
}
},
"min": {
"accounts_per_block": {
"count": 0,
"identifier": {
"hash": "0xef8b9c9a45ef8f2f6f151610de3b766bd07e1c5ccd91e494cde1e8dd37f91778",
"index": 3555327
}
},
"seen_block": 0,
"seen_epoch": 0,
"transaction_per_block": {
"count": 0,
"identifier": {
"hash": "0x4480b45205b1d5555d280f33f97bbf37fe5efc1ecac71b364b2212eb8a84d1e9",
"index": 3767045
}
}
}
}
def select_blocks_by_epoch(blocks, starting, ending):
"""
Selects the blocks starting at the given index (inclusive)
up to the given ending index (exclusive).
"""
def include(blk):
return starting <= blk["block_epoch"] < ending
print(f"selecting block from epoch [{starting}, {ending})")
return select_blocks(blocks, include)
def select_blocks_by_index(blocks, starting, ending, sample=1):
"""
Selects the blocks at every given sample starting at the given epoch (inclusive)
up to the given ending epoch (exclusive).
"""
def include(blk):
index = get_blk_index(blk)
return starting <= index < ending and index % sample == 0
print(f"selecting block from index [{starting}, {ending}) every {sample} samples")
return select_blocks(blocks, include)
def select_blocks(blocks, inclusion_fn=lambda blk: True):
"""
Select all blocks that satisfy the given inclusion function.
"""
return [b for b in blocks if inclusion_fn(b)]
def extract_x_y_col(blocks, x_extractor=lambda b: b["block_epoch"], y_extractor=lambda b: b["amount_transferred"]):
"""
Extracts the x & y col for each block SEQUENTIALLY with the given
extractor functions from the given blocks.
If given extractors return None for some block, it will NOT be added to the col.
"""
x, y = [], []
for blk in blocks:
next_x, next_y = x_extractor(blk), y_extractor(blk)
if next_x is not None:
x.append(next_x)
if next_y is not None:
y.append(next_y)
return x, y
def get_rewards_scatter_plt(blocks, sample=1):
def x_fn(b):
idx = get_blk_index(b)
if idx % sample == 0:
return idx
avg_sum = 0
def y_fn(b):
nonlocal avg_sum
avg_sum += as_one(b["block_rewards"])
if get_blk_index(b) % sample == 0:
to_return, avg_sum = avg_sum, 0
return to_return / sample
idx, val = extract_x_y_col(blocks, x_fn, y_fn)
return go.Scatter(x=idx, y=val, mode='lines', name="blk rewards")
def get_circulating_supply_plt(blocks, sample=1):
def x_fn(b):
idx = get_blk_index(b)
if idx % sample == 0:
return idx
avg_sum = 0
def y_fn(b):
nonlocal avg_sum
avg_sum += as_one(b["approx_circulating_supply_so_far"])
if get_blk_index(b) % sample == 0:
to_return, avg_sum = avg_sum, 0
return to_return / sample
idx, val = extract_x_y_col(blocks, x_fn, y_fn)
return go.Scatter(x=idx, y=val, mode='lines', name="supply")
def get_amount_transferred_plt(blocks, sample=1):
def x_fn(b):
idx = get_blk_index(b)
if idx % sample == 0:
return idx
total = 0
def y_fn(b):
nonlocal total
total += as_one(b["amount_transferred"])
if get_blk_index(b) % sample == 0:
to_return, total = total, 0
return to_return
idx, val = extract_x_y_col(blocks, x_fn, y_fn)
return go.Scatter(x=idx, y=val, mode='lines+markers', name="amount txed")
def get_gas_fees_plt(blocks, sample=1):
def x_fn(b):
idx = get_blk_index(b)
if idx % sample == 0:
return idx
total = 0
def y_fn(b):
nonlocal total
total += as_one(b["gas_fees"])
if get_blk_index(b) % sample == 0:
to_return, total = total, 0
return to_return
idx, val = extract_x_y_col(blocks, x_fn, y_fn)
return go.Scatter(x=idx, y=val, mode='lines+markers', name="gas fees")
def get_cx_in_plt(blocks, sample=1):
def x_fn(b):
idx = get_blk_index(b)
if idx % sample == 0:
return idx
total = 0
def y_fn(b):
nonlocal total
total += as_one(b["cross_shard_amount_received"])
if get_blk_index(b) % sample == 0:
to_return, total = total, 0
return to_return
idx, val = extract_x_y_col(blocks, x_fn, y_fn)
return go.Scatter(x=idx, y=val, mode='lines+markers', name="cx amount in")
def get_cx_out_plt(blocks, sample=1):
def x_fn(b):
idx = get_blk_index(b)
if idx % sample == 0:
return idx
total = 0
def y_fn(b):
nonlocal total
total += as_one(b["cross_shard_amount_sent"])
if get_blk_index(b) % sample == 0:
to_return, total = total, 0
return to_return
idx, val = extract_x_y_col(blocks, x_fn, y_fn)
return go.Scatter(x=idx, y=val, mode='lines+markers', name="cx amount out")
funds_graph_export_path = f"{GRAPH_DIR}/harmony_token_prog_s{SHARD}.html"
def visualize_funds(blocks, sample=1):
fig = go.Figure()
print("calculating rewards...")
fig.add_trace(get_rewards_scatter_plt(blocks, sample))
print("calculating supply...")
fig.add_trace(get_circulating_supply_plt(blocks, sample))
print("calculating amount transferred...")
fig.add_trace(get_amount_transferred_plt(blocks, sample))
print("calculating gas fees...")
fig.add_trace(get_gas_fees_plt(blocks, sample))
print("calculating cx in...")
fig.add_trace(get_cx_in_plt(blocks, sample))
print("calculating cx out...")
fig.add_trace(get_cx_out_plt(blocks, sample))
fig.update_layout(
title=f"ONE Token Progression (shard {SHARD})",
hovermode="x unified",
yaxis_title="ONE Tokens",
xaxis_title="Block Index",
)
clear_output(wait=True)
fig.write_html(funds_graph_export_path)
print(f"exported graph to: {funds_graph_export_path}")
fig.show()
def get_unique_accounts_seen_plt(blocks, sample=1):
def x_fn(b):
idx = get_blk_index(b)
if idx % sample == 0:
return idx
def y_fn(b):
if get_blk_index(b) % sample == 0:
return b["approx_number_of_unique_accounts_so_far"]
idx, val = extract_x_y_col(blocks, x_fn, y_fn)
return go.Scatter(x=idx, y=val, mode='lines', name="unique accs")
def get_accs_per_block_plt(blocks, sample=1):
def x_fn(b):
idx = get_blk_index(b)
if idx % sample == 0:
return idx
avg_sum = 0
def y_fn(b):
nonlocal avg_sum
avg_sum += b["number_of_accounts"]
if get_blk_index(b) % sample == 0:
to_return, avg_sum = avg_sum, 0
return to_return / sample
idx, val = extract_x_y_col(blocks, x_fn, y_fn)
return go.Scatter(x=idx, y=val, mode='lines+markers', name="acc/block")
account_graph_export_path = f"{GRAPH_DIR}/harmony_account_prog_s{SHARD}.html"
def visualize_accounts(blocks, sample=1):
fig = go.Figure()
print("calculating unique accounts...")
fig.add_trace(get_unique_accounts_seen_plt(blocks, sample))
print("calculating accounts per block...")
fig.add_trace(get_accs_per_block_plt(blocks, sample))
fig.update_layout(
title=f"Account Progression (shard {SHARD})",
hovermode="x unified",
yaxis_title="Accounts",
xaxis_title="Block Index",
)
clear_output(wait=True)
fig.write_html(account_graph_export_path)
print(f"exported graph to: {account_graph_export_path}")
fig.show()
def get_contract_creation_plt(blocks, sample=1):
def x_fn(b):
idx = get_blk_index(b)
if idx % sample == 0:
return idx
total = 0
def y_fn(b):
nonlocal total
total += b["number_of_contracts_created"]
if get_blk_index(b) % sample == 0:
to_return, total = total, 0
return to_return
idx, val = extract_x_y_col(blocks, x_fn, y_fn)
return go.Scatter(x=idx, y=val, mode='lines+markers', name="contract creation")
contract_graph_export_path = f"{GRAPH_DIR}/harmony_contract_prog_s{SHARD}.html"
def visualize_contracts(blocks, sample=1):
fig = go.Figure()
print("calculating contract creations...")
fig.add_trace(get_contract_creation_plt(blocks, sample))
fig.update_layout(
title=f"Contract Progression (shard {SHARD})",
xaxis_title="Block Index",
)
clear_output(wait=True)
fig.write_html(contract_graph_export_path)
print(f"exported graph to: {contract_graph_export_path}")
fig.show()
visualize_funds(data["blocks"], 1000)
exported graph to: ./graphs/harmony_token_prog_s0.html
visualize_accounts(data["blocks"], 1000)
exported graph to: ./graphs/harmony_account_prog_s0.html
visualize_contracts(data["blocks"], 1000)
exported graph to: ./graphs/harmony_contract_prog_s0.html
def export_by_epoch(blocks, directory, starting, ending):
"""
Exports the Blocks data to a CSV file in the given Directory.
Starting at the given starting epoch (inclusive) to the ending epoch (exclusive).
"""
def include(blk):
return starting <= blk["block_epoch"] < ending
print(f"exporting from epoch [{starting}, {ending})")
export(blocks, directory, include)
def export_by_block(blocks, directory, starting, ending):
"""
Exports the Blocks data to a CSV file in the given Directory.
Starting at the given starting block (inclusive) to the ending block (exclusive).
"""
def include(blk):
return starting <= blk["block_identifier"]["index"] < ending
print(f"exporting from block [{starting}, {ending})")
export(blocks, directory, include)
def export(blocks, directory, inclusion_fn=lambda blk: True):
"""
Exports all of the given Blocks that pass the given inclusion function
"""
assert os.path.isdir(directory)
save_path = f"{directory}/blocks_export_s{SHARD}.csv"
print(f"exporting to: {save_path}")
with open(save_path, 'w') as f:
f.write("index, hash, epoch, rewards, total_deductions, total_credits, " +
"contracts_deployed, tx_count, acc_count, txed_amount, cx_in, cx_out\n")
for blk in blocks:
if not inclusion_fn(blk):
continue
f.write(f'{blk["block_identifier"]["index"]}, {blk["block_identifier"]["hash"]}, ' +
f'{blk["block_epoch"]}, {blk["block_rewards"]}, {blk["total_amount_deducted_from_accounts"]}, ' +
f'{blk["total_amount_credited_from_accounts"]}, {blk["number_of_contracts_created"]}, ' +
f'{blk["number_of_transactions"]}, {blk["number_of_accounts"]}, ' +
f'{blk["amount_transferred"]}, {blk["cross_shard_amount_received"]}, ' +
f'{blk["cross_shard_amount_sent"]}\n')
clear_output(wait=True)
print(f"finished exporting to {save_path}")
CSV_EXPORT_DIRECTORY = "/tmp"
# export_by_epoch(data["blocks"], CSV_EXPORT_DIRECTORY, 0, data["blocks"][-1]["block_epoch"]+1)
export_by_block(data["blocks"], CSV_EXPORT_DIRECTORY, 0, 1000000)
finished exporting to /tmp/blocks_export_s0.csv